{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41a88443-8207-4fc5-93e0-f86fa62d9f19",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import PyPDFLoader\n",
    "loader = PyPDFLoader(\"../data_base/knowledge_path/pumkin_book/pumpkin_book.pdf\")\n",
    "pages = loader.load()\n",
    "print(pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "104df45c-7b8c-4414-93e7-978c9ec83c5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import WebBaseLoader\n",
    "loader = WebBaseLoader(\"https://zh.d2l.ai/\")\n",
    "docs = loader.load()\n",
    "print(docs[0].page_content[:500])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd66f605-1498-46fc-b568-d04048f52e0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter\n",
    "some_text = \"\"\"When writing documents, writers will use document structure to group content. \\\n",
    "This can convey to the reader, which idea's are related. For example, closely related ideas \\\n",
    "are in sentances. Similar ideas are in paragraphs. Paragraphs form a document. \\n\\n  \\\n",
    "Paragraphs are often delimited with a carriage return or two carriage returns. \\\n",
    "Carriage returns are the \"backslash n\" you see embedded in this string. \\\n",
    "Sentences have a period at the end, but also, have a space.\\\n",
    "and words are separated by space.\"\"\"\n",
    "\n",
    "c_splitter = CharacterTextSplitter(\n",
    "    chunk_size=450,\n",
    "    chunk_overlap=0,\n",
    "    separator = ' '\n",
    ")\n",
    "r_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=450,\n",
    "    chunk_overlap=0, \n",
    "    separators=[\"\\n\\n\", \"\\n\", \" \", \"\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "010f61a8-d74d-47ab-aead-f0ed5b6714e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(c_splitter.split_text(some_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4794c8a-e706-481d-a0c1-d2456448b6c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(r_splitter.split_text(some_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a402885f-f825-44ea-b705-cac83888f09a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://python.langchain.com/api_reference/community/embeddings/langchain_community.embeddings.zhipuai.ZhipuAIEmbeddings.html#langchain_community.embeddings.zhipuai.ZhipuAIEmbeddings\n",
    "\n",
    "from langchain_community.embeddings import ZhipuAIEmbeddings\n",
    "\n",
    "embed = ZhipuAIEmbeddings(\n",
    "    model=\"embedding-2\",\n",
    "    api_key=\"5713143e8fdc4b4a8b284cf97092e70f.qEK71mGIlavzO1Io\",\n",
    ")\n",
    "input_text = \"The meaning of life is 42\"\n",
    "embed.embed_query(input_text)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04dae2fa-8334-40b3-bc43-fb8ef752387f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_chroma import Chroma\n",
    "# 存放文件路径设置\n",
    "persist_directory = r'./langchain'\n",
    "\n",
    "# 创建向量数据库\n",
    "vectordb = Chroma.from_documents(\n",
    "    documents=splits,\n",
    "    embedding=embeddings,\n",
    ")\n",
    "print(vectordb._collection.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a50acf79-03b4-4fc8-9952-b6a5fd2e665f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import WebBaseLoader\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain_community.embeddings import ZhipuAIEmbeddings\n",
    "from langchain_chroma import Chroma\n",
    "# 文件导入\n",
    "loader = WebBaseLoader(\"https://zh.d2l.ai/\")\n",
    "docs = loader.load()\n",
    "\n",
    "# 文本切分\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size = 1500,\n",
    "    chunk_overlap = 150\n",
    ")\n",
    "splits = text_splitter.split_documents(docs)\n",
    "print(len(splits))\n",
    "\n",
    "# 文本嵌入\n",
    "embed = ZhipuAIEmbeddings(\n",
    "    model=\"embedding-2\",\n",
    "    api_key=\"5713143e8fdc4b4a8b284cf97092e70f.qEK71mGIlavzO1Io\",\n",
    ")\n",
    "\n",
    "# 测试\n",
    "# text_1 = \"今天天气不错\"\n",
    "\n",
    "# query_result = embeddings.embed_query(text_1)\n",
    "# print(query_result)\n",
    "\n",
    "# 路径设置\n",
    "persist_directory = r'./langchain-chroma'\n",
    "\n",
    "# 向量库创建\n",
    "vectordb = Chroma.from_documents(\n",
    "    documents=splits,\n",
    "    embedding=embed,\n",
    "    persist_directory=persist_directory\n",
    ")\n",
    "print(vectordb._collection.count())\n",
    "\n",
    "# 检索\n",
    "question = \"图像识别\"\n",
    "docs = vectordb.similarity_search(question,k=3)\n",
    "print(len(docs))\n",
    "print(docs[0].page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db5958ed-fb1c-4759-ad60-fa429858e484",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 假设你有一个 vectordb 对象，并且可以获取所有文档\n",
    "all_docs = vectordb.similarity_search(\"\", k=18)  # 可以使用一个空查询或任意查询返回多个文档\n",
    "\n",
    "# 遍历文档并输出每个文档的元数据\n",
    "for i, doc in enumerate(all_docs):\n",
    "    print(f\"Document {i+1} metadata: {doc.metadata}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48428c53-035e-4792-926d-5e28ac0e8950",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 添加新文档\n",
    "new_loader = WebBaseLoader(\"https://www.deeplearning.ai/\")  # 新文档来源\n",
    "new_docs = new_loader.load()\n",
    "\n",
    "# 文本切分\n",
    "new_splits = text_splitter.split_documents(new_docs)\n",
    "\n",
    "# 添加到现有的向量库\n",
    "vectordb.add_documents(new_splits)\n",
    "\n",
    "# 输出更新后的文档数量\n",
    "print(f\"更新后的文档数量: {vectordb._collection.count()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c1831d0-fa73-42dc-b5d5-123af04d50bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 假设你有一个 vectordb 对象，并且可以获取所有文档\n",
    "all_docs = vectordb.similarity_search(\"\", k=18)  # 可以使用一个空查询或任意查询返回多个文档\n",
    "\n",
    "# 遍历文档并输出每个文档的元数据\n",
    "for i, doc in enumerate(all_docs):\n",
    "    print(f\"Document {i+1} metadata: {doc.metadata}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b8a0193-788a-4b5a-82a0-bb50b8c8c6c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 假设你有一个 vectordb 对象，并且可以获取所有文档\n",
    "all_docs = vectordb.similarity_search(\"\", k=18)  # 可以使用一个空查询或任意查询返回多个文档\n",
    "\n",
    "# 遍历文档并输出每个文档的元数据\n",
    "for i, doc in enumerate(all_docs):\n",
    "    print(f\"Document {i+1} metadata: {doc.metadata}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "824a3dd6-db85-4950-8242-186fde9036c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"大语言模型是怎么推理的？\"\n",
    "docs_meta = vectordb.similarity_search(question, k=3, filter={\"source\": \"https://zh.d2l.ai/\"})\n",
    "print(docs_meta[0].page_content[:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ece7d49-75ad-4fc6-a2c9-6c5aa4568f1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.chat_models import ChatZhipuAI\n",
    "from langchain_core.messages import AIMessage, HumanMessage, SystemMessage\n",
    "from langchain_community.embeddings import BaichuanTextEmbeddings\n",
    "from langchain_chroma import Chroma  # 从 langchain_chroma 中引用 Chroma 类\n",
    "from langchain_community.document_loaders import WebBaseLoader\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "# 文件导入\n",
    "loader = WebBaseLoader(\"https://zh.d2l.ai/\")\n",
    "docs = loader.load()\n",
    "\n",
    "# 文本切分\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size = 1500,\n",
    "    chunk_overlap = 150\n",
    ")\n",
    "splits = text_splitter.split_documents(docs)\n",
    "\n",
    "persist_directory = './langchain-chroma'\n",
    "\n",
    "# 初始化嵌入\n",
    "embed = ZhipuAIEmbeddings(\n",
    "    model=\"embedding-2\",\n",
    "    api_key=\"5713143e8fdc4b4a8b284cf97092e70f.qEK71mGIlavzO1Io\",\n",
    ")\n",
    "\n",
    "# 使用 embedding_function 参数初始化 Chroma\n",
    "vectordb = Chroma(\n",
    "    persist_directory=persist_directory,\n",
    "    embedding_function=embed\n",
    ")\n",
    "\n",
    "chat = ChatZhipuAI(\n",
    "    model=\"glm-4-flash\",\n",
    "    temperature=0.5,\n",
    "    api_key =  \"5713143e8fdc4b4a8b284cf97092e70f.qEK71mGIlavzO1Io\"\n",
    ")\n",
    "\n",
    "from langchain.chains import RetrievalQA\n",
    "question = \"这本书最重要的主题是?\"\n",
    "qa_chain = RetrievalQA.from_chain_type(\n",
    "    chat, # 大模型的选用\n",
    "    retriever=vectordb.as_retriever() # 向量数据库的调用\n",
    ")\n",
    "result = qa_chain({\"query\": question}) # 输入问题\n",
    "print(result[\"result\"]) # 获取回复"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2853565e-6677-4764-b53c-7f845629e291",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "langchain",
   "language": "python",
   "name": "langchain"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
